{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "代码实现了LeNet网络，并完成了手写数字数据集MNIST的训练。\n",
    "'''\n",
    "\n",
    "\n",
    "# 先对MNIST数据集进行读入以及处理\n",
    "# 数据从MNIST官网直接下载\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from struct import unpack\n",
    "\n",
    "def read_image(path):\n",
    "    with open(path, 'rb') as f:\n",
    "        magic, num, rows, cols = unpack('>4I',f.read(16))\n",
    "        img = np.fromfile(f, dtype=np.uint8).reshape(num, rows, cols, 1)   # 将图片格式进行规定，加上通道数\n",
    "    return img\n",
    "\n",
    "def read_label(path):\n",
    "    with open(path, 'rb') as f:\n",
    "        magic, num = unpack('>2I',f.read(8))\n",
    "        label = np.fromfile(f, dtype=np.uint8)\n",
    "    return label\n",
    "\n",
    "def normalize_image(image):\n",
    "    img = image.astype(np.float32)/255.0\n",
    "    return img\n",
    "\n",
    "def one_hot_label(label):\n",
    "    lab = np.zeros((label.size, 10))\n",
    "    for i, row in enumerate(lab):\n",
    "        row[label[i]] = 1\n",
    "    return lab\n",
    "\n",
    "# 加载数据集以及数据预处理\n",
    "\n",
    "def dataset_loader():\n",
    "    train_image = read_image(r'C:\\Users\\95410\\Downloads\\数据集\\MNIST\\train-images.idx3-ubyte')\n",
    "    train_label = read_label(r'C:\\Users\\95410\\Downloads\\数据集\\MNIST\\train-labels.idx1-ubyte')\n",
    "    test_image = read_image(r'C:\\Users\\95410\\Downloads\\数据集\\MNIST\\t10k-images.idx3-ubyte')\n",
    "    test_label = read_label(r'C:\\Users\\95410\\Downloads\\数据集\\MNIST\\t10k-labels.idx1-ubyte')\n",
    "\n",
    "    train_image = normalize_image(train_image)\n",
    "    train_label = one_hot_label(train_label)\n",
    "    train_label = train_label.reshape(train_label.shape[0], train_label.shape[1], 1)\n",
    "\n",
    "    test_image = normalize_image(test_image)\n",
    "    test_label = one_hot_label(test_label)\n",
    "    test_label = test_label.reshape(test_label.shape[0], test_label.shape[1], 1)\n",
    "    \n",
    "    return train_image, train_label, test_image, test_label\n",
    "\n",
    "# image维度为 num×rows×cols×1，像素值范围在0-1\n",
    "# label维度为num×class_num×1\n",
    "train_image, train_label, test_image, test_label = dataset_loader()\n",
    "\n",
    "\n",
    "#由于LeNet的输入应该为32×32，但是MNIST手写数字集图片为28×28，我们需要对它进行零填充。\n",
    "def padding(image, zero_num):\n",
    "    if len(image.shape) == 4:\n",
    "        image_padding = np.zeros((image.shape[0],image.shape[1]+2*zero_num,image.shape[2]+2*zero_num,image.shape[3]))\n",
    "        image_padding[:,zero_num:image.shape[1]+zero_num,zero_num:image.shape[2]+zero_num,:] = image\n",
    "    elif len(image.shape) == 3:\n",
    "        image_padding = np.zeros((image.shape[0]+2*zero_num, image.shape[1]+2*zero_num, image.shape[2]))\n",
    "        image_padding[zero_num:image.shape[0]+zero_num, zero_num:image.shape[1]+zero_num,:] = image\n",
    "    else:\n",
    "        print(\"维度错误\")\n",
    "        sys.exit()\n",
    "    return image_padding\n",
    "\n",
    "train_image = padding(train_image,2)#对初始图像进行零填充，保证与LeNet输入结构一致60000*32*32*1\n",
    "test_image = padding(test_image,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 32, 32, 1)\n",
      "(60000, 10, 1)\n"
     ]
    }
   ],
   "source": [
    "print(train_image.shape)\n",
    "print(train_label.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv(img, conv_filter):\n",
    "   \n",
    "    if len(img.shape)!=3 or len(conv_filter.shape)!=4:\n",
    "        print(\"卷积运算所输入的维度不符合要求\")\n",
    "        sys.exit()\n",
    "        \n",
    "    if img.shape[-1] != conv_filter.shape[-1]:\n",
    "        print(\"卷积输入图片与卷积核的通道数不一致\")\n",
    "        sys.exit()\n",
    "        \n",
    "    img_h, img_w, img_ch = img.shape\n",
    "    filter_num, filter_h, filter_w, img_ch = conv_filter.shape\n",
    "    feature_h = img_h - filter_h + 1\n",
    "    feature_w = img_w - filter_w + 1\n",
    "\n",
    "    # 初始化输出的特征图片，由于没有使用零填充，图片尺寸会减小\n",
    "    img_out = np.zeros((feature_h, feature_w, filter_num))\n",
    "    img_matrix = np.zeros((feature_h*feature_w, filter_h*filter_w*img_ch))\n",
    "    filter_matrix = np.zeros((filter_h*filter_w*img_ch, filter_num))\n",
    "    \n",
    "    # 将输入图片张量转换成矩阵形式\n",
    "    for i in range(feature_h*feature_w):\n",
    "        for j in range(img_ch):\n",
    "            img_matrix[i, j*filter_h*filter_w:(j+1)*filter_h*filter_w] = \\\n",
    "            img[np.uint16(i/feature_w):np.uint16(i/feature_w+filter_h),np.uint16(i%feature_w):np.uint16(i%feature_w+filter_w),j].reshape(filter_h*filter_w)\n",
    "    \n",
    "    # 将卷积核张量转换成矩阵形式\n",
    "    for i in range(filter_num):\n",
    "        filter_matrix[:,i] = conv_filter[i,:].reshape(filter_w*filter_h*img_ch)\n",
    "    \n",
    "    feature_matrix = np.dot(img_matrix, filter_matrix)\n",
    "    \n",
    "    # 将以矩阵形式存储的卷积结果再转换为张量形式\n",
    "    for i in range(filter_num):\n",
    "        img_out[:,:,i] = feature_matrix[:,i].reshape(feature_w, feature_h)\n",
    "    \n",
    "    return img_out\n",
    "\n",
    "\n",
    "def conv_(img, conv_filter):\n",
    "    \n",
    "    # 对二维图像以及二维卷积核进行卷积，不填充\n",
    "    img_h, img_w = img.shape\n",
    "    filter_h, filter_w = conv_filter.shape\n",
    "    feature_h = img_h - filter_h + 1\n",
    "    feature_w = img_w - filter_w + 1\n",
    "    \n",
    "    img_matrix = np.zeros((feature_h*feature_w, filter_h*filter_w))\n",
    "    for i in range(feature_h*feature_w):\n",
    "        img_matrix[i:] = \\\n",
    "        img[np.uint16(i/feature_w):np.uint16(i/feature_w+filter_h),np.uint16(i%feature_w):np.uint16(i%feature_w+filter_w)].reshape(filter_w*filter_h)\n",
    "    filter_matrix = conv_filter.reshape(filter_h*filter_w,1)\n",
    "    \n",
    "    img_out = np.dot(img_matrix, filter_matrix)\n",
    "    \n",
    "    img_out = img_out.reshape(feature_h,feature_w)\n",
    "            \n",
    "    return img_out\n",
    "\n",
    "\n",
    "def conv_cal_w(out_img_delta, in_img):\n",
    "    #由卷积前的图片以及卷积后的delta计算卷积核的梯度\n",
    "    nabla_conv = np.zeros([out_img_delta.shape[-1], in_img.shape[0]-out_img_delta.shape[0]+1, \n",
    "                           in_img.shape[1]-out_img_delta.shape[1]+1, in_img.shape[-1]])\n",
    "    for filter_num in range(nabla_conv.shape[0]):\n",
    "        for ch_num in range(nabla_conv.shape[-1]):\n",
    "            nabla_conv[filter_num,:,:,ch_num] = conv_(in_img[:,:,ch_num], out_img_delta[:,:,filter_num])\n",
    "    return nabla_conv\n",
    "\n",
    "def conv_cal_b(out_img_delta):\n",
    "    nabla_b = np.zeros((out_img_delta.shape[-1],1))\n",
    "    for i in range(out_img_delta.shape[-1]):\n",
    "        nabla_b[i] = np.sum(out_img_delta[:,:,i])\n",
    "    return nabla_b\n",
    "\n",
    "\n",
    "def relu(feature):\n",
    "    '''Relu激活函数，有两种情况会使用到\n",
    "    当在卷积层中使用时，feature为一个三维张量，，[行，列，通道]\n",
    "    当在全连接层中使用时，feature为一个列向量'''\n",
    "    relu_out = np.zeros(feature.shape)\n",
    "    \n",
    "    if len(feature.shape) > 2:\n",
    "        for ch_num in range(feature.shape[-1]):\n",
    "            for r in range(feature.shape[0]):\n",
    "                for c in range(feature.shape[1]):\n",
    "                    relu_out[r,c,ch_num] = max(feature[r,c,ch_num], 0)\n",
    "    else:\n",
    "        for r in range(feature.shape[0]):\n",
    "            relu_out[r,0] = max(feature[r,0], 0)\n",
    "    return relu_out\n",
    "\n",
    "\n",
    "def relu_prime(feature):  # 对relu函数的求导\n",
    "    '''relu函数的一阶导数，间断点导数认为是0'''\n",
    "    \n",
    "    relu_prime_out = np.zeros(feature.shape)\n",
    "    \n",
    "    if len(feature.shape) > 2:\n",
    "        for ch_num in range(feature.shape[-1]):\n",
    "            for r in range(feature.shape[0]):\n",
    "                for c in range(feature.shape[1]):\n",
    "                    if feature[r,c,ch_num] > 0:\n",
    "                        relu_prime_out[r,c,ch_num] = 1\n",
    "\n",
    "    else:\n",
    "        for r in range(feature.shape[0]):\n",
    "            if feature[r,0] > 0:\n",
    "                relu_prime_out[r, 0] = 1\n",
    "            \n",
    "    return relu_prime_out\n",
    "\n",
    "\n",
    "def pool(feature, size=2, stride=2):\n",
    "    '''最大池化操作\n",
    "    同时输出池化后的结果以及用于记录最大位置的张量，方便之后delta误差反向传播'''\n",
    "    pool_out = np.zeros([np.uint16((feature.shape[0]-size)/stride+1),\n",
    "                        np.uint16((feature.shape[1]-size)/stride+1),\n",
    "                         feature.shape[2]])\n",
    "    pool_out_max_location = np.zeros(pool_out.shape) # 该函数用于记录最大值位置\n",
    "    for ch_num in range(feature.shape[-1]):\n",
    "        r_out = 0\n",
    "        for r in np.arange(0, feature.shape[0]-size+1, stride):\n",
    "            c_out = 0\n",
    "            for c in np.arange(0, feature.shape[1]-size+1, stride):\n",
    "                pool_out[r_out, c_out, ch_num] = np.max(feature[r:r+size, c:c+size, ch_num])\n",
    "                pool_out_max_location[r_out, c_out, ch_num] = np.argmax(feature[r:r+size, c:c+size, ch_num])  #记录最大点位置\n",
    "                c_out += 1\n",
    "            r_out += 1\n",
    "    return pool_out, pool_out_max_location\n",
    "\n",
    "def pool_delta_error_bp(pool_out_delta, pool_out_max_location, size=2, stride=2):\n",
    "    \n",
    "    delta = np.zeros([np.uint16((pool_out_delta.shape[0]-1)*stride+size),\n",
    "                      np.uint16((pool_out_delta.shape[1]-1)*stride+size),\n",
    "                      pool_out_delta.shape[2]])\n",
    "    for ch_num in range(pool_out_delta.shape[-1]):\n",
    "        for r in range(pool_out_delta.shape[0]):\n",
    "            for c in range(pool_out_delta.shape[1]):\n",
    "                order = pool_out_max_location[r,c,ch_num]\n",
    "                m = np.uint16(stride*r+order//size)  \n",
    "                n = np.uint16(stride*c+order%size)\n",
    "                delta[m, n, ch_num] = pool_out_delta[r,c,ch_num]\n",
    "    return delta\n",
    "\n",
    "def rot180(conv_filters):\n",
    "    rot180_filters = np.zeros((conv_filters.shape))\n",
    "    for filter_num in range(conv_filters.shape[0]):\n",
    "        for img_ch in range(conv_filters.shape[-1]):\n",
    "            rot180_filters[filter_num,:,:,img_ch] = np.flipud(np.fliplr(conv_filters[filter_num,:,:,img_ch]))\n",
    "    return rot180_filters\n",
    "                \n",
    "    \n",
    "def soft_max(z):\n",
    "        \n",
    "    tmp = np.max(z)\n",
    "    z -= tmp  # 用于缩放每行的元素，避免溢出，有效\n",
    "    z = np.exp(z)\n",
    "    tmp = np.sum(z)\n",
    "    z /= tmp\n",
    "    \n",
    "    return z\n",
    "\n",
    "def add_bias(conv, bias):\n",
    "    if conv.shape[-1] != bias.shape[0]:\n",
    "        print(\"给卷积添加偏置维度出错\")\n",
    "    else:\n",
    "        for i in range(bias.shape[0]):\n",
    "            conv[:,:,i] += bias[i,0]\n",
    "    return conv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        \n",
    "        '''\n",
    "        2层卷积，2层池化，3层全连接'''\n",
    "        self.filters = [np.random.randn(6, 5, 5, 1)] #图像变成 28*28*6 池化后图像变成14*14*6\n",
    "        self.filters_biases = [np.random.randn(6,1)]\n",
    "        self.filters.append(np.random.randn(16, 5, 5, 6)) #图像变成 10*10*16 池化后变成5*5*16\n",
    "        self.filters_biases.append(np.random.randn(16,1))\n",
    "        \n",
    "        self.weights = [np.random.randn(120,400)]\n",
    "        self.weights.append(np.random.randn(84,120))\n",
    "        self.weights.append(np.random.randn(10,84))\n",
    "        self.biases = [np.random.randn(120,1)]\n",
    "        self.biases.append(np.random.randn(84,1))\n",
    "        self.biases.append(np.random.randn(10,1))\n",
    "        \n",
    "    \n",
    "    \n",
    "    def feed_forward(self, x):\n",
    "        #第一层卷积\n",
    "        conv1 = add_bias( conv(x, self.filters[0]), self.filters_biases[0] )\n",
    "        relu1 = relu(conv1)\n",
    "        pool1, pool1_max_locate = pool(relu1)\n",
    "        \n",
    "        #第二层卷积\n",
    "        conv2 = add_bias( conv(pool1, self.filters[1]), self.filters_biases[1])\n",
    "        relu2 = relu(conv2)\n",
    "        pool2, pool2_max_locate = pool(relu2)\n",
    "        \n",
    "        #拉直\n",
    "        straight_input = pool2.reshape(pool2.shape[0] * pool2.shape[1] * pool2.shape[2], 1)\n",
    "        \n",
    "        #第一层全连接\n",
    "        full_connect1_z = np.dot(self.weights[0], straight_input) + self.biases[0]\n",
    "        full_connect1_a = relu(full_connect1_z)\n",
    "        \n",
    "        #第二层全连接\n",
    "        full_connect2_z = np.dot(self.weights[1], full_connect1_a) + self.biases[1]\n",
    "        full_connect2_a = relu(full_connect2_z)\n",
    "        \n",
    "        #第三层全连接（输出）\n",
    "        full_connect3_z = np.dot(self.weights[2], full_connect2_a) + self.biases[2]\n",
    "        full_connect3_a = soft_max(full_connect3_z)\n",
    "        return full_connect3_a\n",
    "    \n",
    "    def evaluate(self, images, labels):\n",
    "        result = 0\n",
    "        for img, lab in zip(images, labels):\n",
    "            predict_label = self.feed_forward(img)\n",
    "            if np.argmax(predict_label) == np.argmax(lab):\n",
    "                result += 1\n",
    "        return result\n",
    "    \n",
    "    def SGD(self, train_image, train_label, epochs, mini_batch_size, eta):\n",
    "        '''Stochastic gradiend descent'''\n",
    "        batch_num = 0\n",
    "        for j in range(epochs):\n",
    "           \n",
    "            mini_batches_image = [train_image[k:k+mini_batch_size] for k in range(0, len(train_image), mini_batch_size)]\n",
    "            mini_batches_label = [train_label[k:k+mini_batch_size] for k in range(0, len(train_label), mini_batch_size)]\n",
    "            for mini_batch_image, mini_batch_label in zip(mini_batches_image, mini_batches_label):\n",
    "                batch_num += 1\n",
    "                if batch_num * mini_batch_size > len(train_image):\n",
    "                    batch_num = 1\n",
    "                \n",
    "                self.update_mini_batch(mini_batch_image, mini_batch_label, eta, mini_batch_size)\n",
    "                \n",
    "                #if batch_num % 100 == 0:\n",
    "                    #print(\"after {0} training batch: accuracy is {1}/{2}\".format(batch_num, self.evaluate(train_image[0:1000], train_label[0:1000]), len(train_image[0:1000])))\n",
    "                \n",
    "                print(\"\\rEpoch{0}:{1}/{2}\".format(j+1, batch_num*mini_batch_size, len(train_image)), end=' ')\n",
    "            \n",
    "            print(\"After epoch{0}: accuracy is {1}/{2}\".format(j+1, self.evaluate(test_image, test_label), len(test_image)))\n",
    "                \n",
    "    def update_mini_batch(self, mini_batch_image, mini_batch_label, eta, mini_batch_size):\n",
    "        '''通过一个batch的数据对神经网络参数进行更新\n",
    "        需要先求这个batch中每张图片的误差反向传播求得的权重梯度以及偏置梯度'''\n",
    "        \n",
    "        nabla_b = [np.zeros(b.shape) for b in self.biases]\n",
    "        nabla_w = [np.zeros(w.shape) for w in self.weights]\n",
    "        \n",
    "        nabla_f = [np.zeros(f.shape) for f in self.filters]\n",
    "        nabla_fb = [np.zeros(fb.shape) for fb in self.filters_biases]\n",
    "        \n",
    "        for x,y in zip(mini_batch_image, mini_batch_label):\n",
    "            delta_nabla_w, delta_nabla_b, delta_nabla_f, delta_nabla_fb = self.backprop(x, y)\n",
    "            nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]\n",
    "            nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]\n",
    "            nabla_f = [nf+dnf for nf, dnf in zip(nabla_f, delta_nabla_f)]\n",
    "            nabla_fb = [nfb + dnfb for nfb, dnfb in zip(nabla_fb, delta_nabla_fb)]\n",
    "        self.weights = [w-(eta/mini_batch_size)*nw for w, nw in zip(self.weights, nabla_w)]\n",
    "        self.biases = [b-(eta/mini_batch_size)*nb for b, nb in zip(self.biases, nabla_b)]\n",
    "        self.filters = [f-(eta/mini_batch_size)*nf for f, nf in zip(self.filters, nabla_f)]\n",
    "        self.filters_biases = [fb-(eta/mini_batch_size)*nfb for fb, nfb in zip(self.filters_biases, nabla_fb)]\n",
    "    \n",
    "    def backprop(self, x, y):\n",
    "        \n",
    "        '''计算通过单幅图像求得梯度'''\n",
    "        \n",
    "        #先前向传播，求出各中间量\n",
    "        #第一层卷积\n",
    "        conv1 = add_bias( conv(x, self.filters[0]), self.filters_biases[0] )\n",
    "        relu1 = relu(conv1)\n",
    "        pool1, pool1_max_locate = pool(relu1)\n",
    "\n",
    "        #第二层卷积\n",
    "        conv2 = add_bias( conv(pool1, self.filters[1]), self.filters_biases[1] )\n",
    "        relu2 = relu(conv2)\n",
    "        pool2, pool2_max_locate = pool(relu2)\n",
    "        \n",
    "        #拉直\n",
    "        straight_input = pool2.reshape(pool2.shape[0] * pool2.shape[1] * pool2.shape[2], 1)\n",
    "        \n",
    "        #第一层全连接\n",
    "        full_connect1_z = np.dot(self.weights[0], straight_input) + self.biases[0]\n",
    "        full_connect1_a = relu(full_connect1_z)\n",
    "        \n",
    "        #第二层全连接\n",
    "        full_connect2_z = np.dot(self.weights[1], full_connect1_a) + self.biases[1]\n",
    "        full_connect2_a = relu(full_connect2_z)\n",
    "        \n",
    "        #第三层全连接（输出）\n",
    "        full_connect3_z = np.dot(self.weights[2], full_connect2_a) + self.biases[2]\n",
    "        full_connect3_a = soft_max(full_connect3_z)\n",
    "            \n",
    "        # 在这里我们使用交叉熵损失，激活函数为softmax，因此delta值就为 a-y，即对正确位置的预测值减1\n",
    "        delta_fc3 = full_connect3_a - y\n",
    "        delta_fc2 = np.dot(self.weights[2].transpose(), delta_fc3) * relu_prime(full_connect2_z)\n",
    "        delta_fc1 = np.dot(self.weights[1].transpose(), delta_fc2) * relu_prime(full_connect1_z)\n",
    "        delta_straight_input = np.dot(self.weights[0].transpose(), delta_fc1)# 这里没有激活函数？\n",
    "        delta_pool2 = delta_straight_input.reshape(pool2.shape)\n",
    "        \n",
    "        delta_conv2 = pool_delta_error_bp(delta_pool2, pool2_max_locate) * relu_prime(conv2)\n",
    "        \n",
    "        delta_pool1 = conv(padding(delta_conv2, self.filters[1].shape[1]-1), rot180(self.filters[1]).swapaxes(0,3))\n",
    "        \n",
    "        delta_conv1 = pool_delta_error_bp(delta_pool1, pool1_max_locate) * relu_prime(conv1)\n",
    "        \n",
    "        \n",
    "        \n",
    "        #求各参数的导数\n",
    "        nabla_w2 = np.dot(delta_fc3, full_connect2_a.transpose())\n",
    "        nabla_b2 = delta_fc3\n",
    "        nabla_w1 = np.dot(delta_fc2, full_connect1_a.transpose())\n",
    "        nabla_b1 = delta_fc2\n",
    "        nabla_w0 = np.dot(delta_fc1, straight_input.transpose())\n",
    "        nabla_b0 = delta_fc1\n",
    "        \n",
    "        \n",
    "        nabla_filters1 = conv_cal_w(delta_conv2, pool1) \n",
    "        nabla_filters0 = conv_cal_w(delta_conv1, x)\n",
    "        nabla_filters_biases1 = conv_cal_b(delta_conv2)\n",
    "        nabla_filters_biases0 = conv_cal_b(delta_conv1)\n",
    "        \n",
    "        nabla_w = [nabla_w0, nabla_w1, nabla_w2]\n",
    "        nabla_b = [nabla_b0, nabla_b1, nabla_b2]\n",
    "        nabla_f = [nabla_filters0, nabla_filters1]\n",
    "        nabla_fb = [nabla_filters_biases0, nabla_filters_biases1]\n",
    "        return nabla_w, nabla_b, nabla_f, nabla_fb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch1:60000/60000 After epoch1: accuracy is 8544/10000\n",
      "Epoch2:60000/60000 After epoch2: accuracy is 8789/10000\n",
      "Epoch3:60000/60000 After epoch3: accuracy is 8920/10000\n",
      "Epoch4:60000/60000 After epoch4: accuracy is 8906/10000\n",
      "Epoch5:60000/60000 After epoch5: accuracy is 8718/10000\n",
      "Epoch6:60000/60000 After epoch6: accuracy is 8634/10000\n",
      "Epoch7:60000/60000 After epoch7: accuracy is 8301/10000\n",
      "Epoch8:20460/60000 "
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-33-500975fa7c99>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mnet\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mConvNet\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mnet\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSGD\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_image\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_label\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m30\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m10\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m3e-5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-32-3aba1e45b726>\u001b[0m in \u001b[0;36mSGD\u001b[1;34m(self, train_image, train_label, epochs, mini_batch_size, eta)\u001b[0m\n\u001b[0;32m     66\u001b[0m                     \u001b[0mbatch_num\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     67\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 68\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate_mini_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmini_batch_image\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmini_batch_label\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmini_batch_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     69\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     70\u001b[0m                 \u001b[1;31m#if batch_num % 100 == 0:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-32-3aba1e45b726>\u001b[0m in \u001b[0;36mupdate_mini_batch\u001b[1;34m(self, mini_batch_image, mini_batch_label, eta, mini_batch_size)\u001b[0m\n\u001b[0;32m     86\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     87\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmini_batch_image\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmini_batch_label\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 88\u001b[1;33m             \u001b[0mdelta_nabla_w\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdelta_nabla_b\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdelta_nabla_f\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdelta_nabla_fb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackprop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     89\u001b[0m             \u001b[0mnabla_b\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mnb\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mdnb\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mnb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdnb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnabla_b\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdelta_nabla_b\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     90\u001b[0m             \u001b[0mnabla_w\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mnw\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mdnw\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mnw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdnw\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnabla_w\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdelta_nabla_w\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-32-3aba1e45b726>\u001b[0m in \u001b[0;36mbackprop\u001b[1;34m(self, x, y)\u001b[0m\n\u001b[0;32m    109\u001b[0m         \u001b[0mconv2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0madd_bias\u001b[0m\u001b[1;33m(\u001b[0m \u001b[0mconv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpool1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfilters\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfilters_biases\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    110\u001b[0m         \u001b[0mrelu2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrelu\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconv2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 111\u001b[1;33m         \u001b[0mpool2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpool2_max_locate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpool\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrelu2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    112\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    113\u001b[0m         \u001b[1;31m#拉直\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-31-0937391323d0>\u001b[0m in \u001b[0;36mpool\u001b[1;34m(feature, size, stride)\u001b[0m\n\u001b[0;32m    124\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeature\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstride\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    125\u001b[0m                 \u001b[0mpool_out\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mr_out\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mc_out\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mch_num\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeature\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mr\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mc\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mc\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mch_num\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 126\u001b[1;33m                 \u001b[0mpool_out_max_location\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mr_out\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mc_out\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mch_num\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeature\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mr\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mc\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mc\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mch_num\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m#记录最大点位置\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    127\u001b[0m                 \u001b[0mc_out\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    128\u001b[0m             \u001b[0mr_out\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\numpy\\core\\fromnumeric.py\u001b[0m in \u001b[0;36margmax\u001b[1;34m(a, axis, out)\u001b[0m\n\u001b[0;32m   1101\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1102\u001b[0m     \"\"\"\n\u001b[1;32m-> 1103\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_wrapfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'argmax'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1104\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1105\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\numpy\\core\\fromnumeric.py\u001b[0m in \u001b[0;36m_wrapfunc\u001b[1;34m(obj, method, *args, **kwds)\u001b[0m\n\u001b[0;32m     54\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_wrapfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     55\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 56\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     57\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[1;31m# An AttributeError occurs if the object does not have\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "net = ConvNet()\n",
    "net.SGD(train_image, train_label, 30, 10, 3e-5) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
